{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot import Collection\n",
    "gpt_4 = Collection.from_json(\"thoughtsource_100_openai_chat_gpt-4_None_zhou-01.json\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '9889e5389917d812c09d6e5d382d333d', 'ref_id': '', 'question': 'After he got hired he hoped for success at his what?', 'type': 'multiplechoice', 'choices': ['vocation', 'new job', 'michigan', 'working hard', 'manual'], 'context': '', 'cot': ['He got hired at a new job.', 'A person hope for success after being hired at his new job.', 'A person might not be hired in a vocation.', 'At michigan, a person might not be hired.', 'Working hard or manual isnt a place where a person can be hired.'], 'answer': ['new job'], 'generated_cot': [{'id': '3856fcec-47d0-4ad9-832b-c64b4852e1e2', 'fragments_version': '0.01', 'instruction': None, 'cot_trigger': 'zhou-01', 'cot_trigger_template': '{instruction}\\n\\n{question}\\n{answer_choices}\\n\\n{cot_trigger}', 'prompt_text': '', 'cot': 'A) vocation - This could be correct, as it refers to his chosen profession or occupation.\\nB) new job - This is also correct, as it directly refers to the position he was hired for.\\nC) michigan - This is not correct, as it is a location and not related to his success in a job.\\nD) working hard - This is not correct, as it is an action and not a specific job or position.\\nE) manual - This is not correct, as it refers to a type of work or instruction book, not a job or position.\\n\\nBased on the context, the best answer is B) new job.', 'answers': [{'id': '429e87f6-b02a-4409-8780-b60c4b611179', 'answer_extraction': 'kojima-A-E', 'answer_extraction_template': '{instruction}\\n\\n{question}\\n{answer_choices}\\n\\n{cot_trigger}{cot}\\n{answer_extraction}', 'answer_extraction_text': '', 'answer': 'B) new job', 'correct_answer': True}], 'author': 'thoughtsource', 'date': '2023/03/17 10:01:28', 'api_service': 'openai_chat', 'model': \"{'name': 'gpt-4', 'temperature': 0, 'max_tokens': 512}\", 'comment': '', 'annotations': []}], 'feedback': []}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4['commonsense_qa']['validation'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt_4.select_generated_cots(answers=\"False\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'gpt-4': {'None_zhou-01_kojima-A-E': 0.0}}}}, 'med_qa': {'test': {'accuracy': {'gpt-4': {'None_zhou-01_kojima-A-E': 0.0}}}}, 'medmc_qa': {'validation': {'accuracy': {'gpt-4': {'None_zhou-01_kojima-A-D': 0.0}}}}, 'open_book_qa': {'test': {'accuracy': {'gpt-4': {'None_zhou-01_kojima-A-D': 0.0}}}}, 'strategy_qa': {'train': {'accuracy': {'gpt-4': {'None_zhou-01_kojima-yes-no': 0.0}}}}, 'worldtree': {'test': {'accuracy': {'gpt-4': {'None_zhou-01_kojima-A-D': 0.0}}}}}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '9889e5389917d812c09d6e5d382d333d', 'ref_id': '', 'question': 'After he got hired he hoped for success at his what?', 'type': 'multiplechoice', 'choices': ['vocation', 'new job', 'michigan', 'working hard', 'manual'], 'context': '', 'cot': ['He got hired at a new job.', 'A person hope for success after being hired at his new job.', 'A person might not be hired in a vocation.', 'At michigan, a person might not be hired.', 'Working hard or manual isnt a place where a person can be hired.'], 'answer': ['new job'], 'generated_cot': [], 'feedback': []}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4['commonsense_qa']['validation'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt_4.dump(\"false_gpt_4.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 100     | -      |\n",
       "| med_qa         | -       | -       | 100    |\n",
       "| medmc_qa       | -       | 100     | -      |\n",
       "| open_book_qa   | -       | -       | 100    |\n",
       "| strategy_qa    | 100     | -       | -      |\n",
       "| worldtree      | -       | -       | 100    |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt_4.unload_datasets([\"med_qa\",\"medmc_qa\",\"open_book_qa\",\"strategy_qa\",\"worldtree\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   |   Valid | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       |     100 | -      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'medmc_qa', 'open_book_qa', 'pubmed_qa', 'qed', 'strategy_qa', 'svamp', 'worldtree']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt_4.dump(\"false_gpt_4_cs.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['id', 'ref_id', 'question', 'type', 'choices', 'context', 'cot', 'answer', 'generated_cot', 'feedback'],\n",
       "    num_rows: 1\n",
       "})"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs_gp = gpt_4['commonsense_qa']['validation']\n",
    "\n",
    "cs_gp.filter(lambda x: x[\"id\"] == \"5ea6b94d1a911365b06cf776919413e8\") #takes first one\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'b8c0a4703079cf661d7261a60a1bcbff'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs_gp[0]['id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': '5ea6b94d1a911365b06cf776919413e8', 'ref_id': '', 'question': 'What does \\tdrinking alcohol lead to?', 'type': 'multiplechoice', 'choices': ['have fun', 'intoxication', 'vomiting', 'drinking more alcohol', 'nausea'], 'context': '', 'cot': [\"Drinking alcohol lead to intoxication as alcohol make you intoxicated.drinking alcohol doesn't lead to having fun,vomiting,drinking more alcohol and nausea always.\"], 'answer': ['intoxication'], 'generated_cot': [{'id': 'a66a4277-9e3e-4eed-ada3-9e75d9654d21', 'fragments_version': '0.01', 'instruction': None, 'cot_trigger': 'zhou-01', 'cot_trigger_template': '{instruction}\\n\\n{question}\\n{answer_choices}\\n\\n{cot_trigger}', 'prompt_text': '', 'cot': \"Drinking alcohol can lead to various outcomes depending on the individual and the amount consumed. Here are the possible outcomes for each option:\\n\\nA) have fun - Drinking alcohol can lead to having fun for some people, as it may help them relax and enjoy social situations.\\n\\nB) intoxication - Drinking alcohol can lead to intoxication, which is the state of being under the influence of alcohol. This can impair a person's judgment, coordination, and reaction time.\\n\\nC) vomiting - Drinking excessive amounts of alcohol can lead to vomiting, as the body tries to expel the toxins.\\n\\nD) drinking more alcohol - Drinking alcohol can sometimes lead to a desire to drink more alcohol, especially in social situations or for individuals who struggle with alcohol addiction.\\n\\nE) nausea - Drinking alcohol can cause nausea, especially if consumed in large quantities or on an empty stomach.\\n\\nSo, the correct answer is that drinking alcohol can lead to all of these outcomes (A, B, C, D, and E), depending on the individual and the amount consumed.\", 'answers': [{'id': '7745a792-44b3-4214-a38a-8164385b69ac', 'answer_extraction': 'kojima-A-E', 'answer_extraction_template': '{instruction}\\n\\n{question}\\n{answer_choices}\\n\\n{cot_trigger}{cot}\\n{answer_extraction}', 'answer_extraction_text': '', 'answer': 'all of the above (A, B, C, D, and E).', 'correct_answer': False}], 'author': 'thoughtsource', 'date': '2023/03/17 10:53:22', 'api_service': 'openai_chat', 'model': \"{'name': 'gpt-4', 'temperature': 0, 'max_tokens': 512}\", 'comment': '', 'annotations': []}], 'feedback': []}\n",
      "89\n"
     ]
    }
   ],
   "source": [
    "cs_gp = gpt_4['commonsense_qa']['validation']\n",
    "i = 0\n",
    "for element in cs_gp:\n",
    "    if element['id'] == \"5ea6b94d1a911365b06cf776919413e8\":\n",
    "        print(element)\n",
    "        print(i)\n",
    "    i += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "alcohol_dataset = cs_gp.select([89])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2226"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alcohol_dataset.to_json(\"alcohol_dataset\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "Couldn't find a dataset script at /Users/robertpraas/Desktop/ThoughtSource/libs/cot/cot/datasets/id/id.py",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/Users/robertpraas/Desktop/ThoughtSource/notebooks/gpt_4_book.ipynb Cell 16\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/gpt_4_book.ipynb#X23sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m alcohol \u001b[39m=\u001b[39m Collection\u001b[39m.\u001b[39;49mfrom_json(\u001b[39m\"\u001b[39;49m\u001b[39malcohol_dataset.json\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n",
      "File \u001b[0;32m~/Desktop/ThoughtSource/libs/cot/cot/dataloader.py:269\u001b[0m, in \u001b[0;36mCollection.from_json\u001b[0;34m(path_or_json, download_mode, source)\u001b[0m\n\u001b[1;32m    267\u001b[0m collection \u001b[39m=\u001b[39m Collection()\n\u001b[1;32m    268\u001b[0m \u001b[39mfor\u001b[39;00m dataset_name \u001b[39min\u001b[39;00m content\u001b[39m.\u001b[39mkeys():\n\u001b[0;32m--> 269\u001b[0m     info \u001b[39m=\u001b[39m ds\u001b[39m.\u001b[39;49mload_dataset_builder(\n\u001b[1;32m    270\u001b[0m         \u001b[39mstr\u001b[39;49m(scripts[dataset_name]), name\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39msource\u001b[39;49m\u001b[39m\"\u001b[39;49m \u001b[39mif\u001b[39;49;00m source \u001b[39melse\u001b[39;49;00m \u001b[39m\"\u001b[39;49m\u001b[39mthoughtsource\u001b[39;49m\u001b[39m\"\u001b[39;49m, download_mode\u001b[39m=\u001b[39;49mdownload_mode\n\u001b[1;32m    271\u001b[0m     )\u001b[39m.\u001b[39minfo\n\u001b[1;32m    272\u001b[0m     dataset_dict \u001b[39m=\u001b[39m \u001b[39mdict\u001b[39m()\n\u001b[1;32m    273\u001b[0m     \u001b[39mfor\u001b[39;00m split_name \u001b[39min\u001b[39;00m content[dataset_name]\u001b[39m.\u001b[39mkeys():\n",
      "File \u001b[0;32m/opt/anaconda3/lib/python3.9/site-packages/datasets/load.py:1490\u001b[0m, in \u001b[0;36mload_dataset_builder\u001b[0;34m(path, name, data_dir, data_files, cache_dir, features, download_config, download_mode, revision, use_auth_token, **config_kwargs)\u001b[0m\n\u001b[1;32m   1488\u001b[0m     download_config \u001b[39m=\u001b[39m download_config\u001b[39m.\u001b[39mcopy() \u001b[39mif\u001b[39;00m download_config \u001b[39melse\u001b[39;00m DownloadConfig()\n\u001b[1;32m   1489\u001b[0m     download_config\u001b[39m.\u001b[39muse_auth_token \u001b[39m=\u001b[39m use_auth_token\n\u001b[0;32m-> 1490\u001b[0m dataset_module \u001b[39m=\u001b[39m dataset_module_factory(\n\u001b[1;32m   1491\u001b[0m     path,\n\u001b[1;32m   1492\u001b[0m     revision\u001b[39m=\u001b[39;49mrevision,\n\u001b[1;32m   1493\u001b[0m     download_config\u001b[39m=\u001b[39;49mdownload_config,\n\u001b[1;32m   1494\u001b[0m     download_mode\u001b[39m=\u001b[39;49mdownload_mode,\n\u001b[1;32m   1495\u001b[0m     data_dir\u001b[39m=\u001b[39;49mdata_dir,\n\u001b[1;32m   1496\u001b[0m     data_files\u001b[39m=\u001b[39;49mdata_files,\n\u001b[1;32m   1497\u001b[0m )\n\u001b[1;32m   1499\u001b[0m \u001b[39m# Get dataset builder class from the processing script\u001b[39;00m\n\u001b[1;32m   1500\u001b[0m builder_cls \u001b[39m=\u001b[39m import_main_class(dataset_module\u001b[39m.\u001b[39mmodule_path)\n",
      "File \u001b[0;32m/opt/anaconda3/lib/python3.9/site-packages/datasets/load.py:1164\u001b[0m, in \u001b[0;36mdataset_module_factory\u001b[0;34m(path, revision, download_config, download_mode, force_local_path, dynamic_modules_path, data_dir, data_files, **download_kwargs)\u001b[0m\n\u001b[1;32m   1160\u001b[0m         \u001b[39mreturn\u001b[39;00m LocalDatasetModuleFactoryWithScript(\n\u001b[1;32m   1161\u001b[0m             path, download_mode\u001b[39m=\u001b[39mdownload_mode, dynamic_modules_path\u001b[39m=\u001b[39mdynamic_modules_path\n\u001b[1;32m   1162\u001b[0m         )\u001b[39m.\u001b[39mget_module()\n\u001b[1;32m   1163\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1164\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mFileNotFoundError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCouldn\u001b[39m\u001b[39m'\u001b[39m\u001b[39mt find a dataset script at \u001b[39m\u001b[39m{\u001b[39;00mrelative_to_absolute_path(path)\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m   1165\u001b[0m \u001b[39melif\u001b[39;00m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39misfile(combined_path):\n\u001b[1;32m   1166\u001b[0m     \u001b[39mreturn\u001b[39;00m LocalDatasetModuleFactoryWithScript(\n\u001b[1;32m   1167\u001b[0m         combined_path, download_mode\u001b[39m=\u001b[39mdownload_mode, dynamic_modules_path\u001b[39m=\u001b[39mdynamic_modules_path\n\u001b[1;32m   1168\u001b[0m     )\u001b[39m.\u001b[39mget_module()\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: Couldn't find a dataset script at /Users/robertpraas/Desktop/ThoughtSource/libs/cot/cot/datasets/id/id.py"
     ]
    }
   ],
   "source": [
    "alcohol = Collection.from_json(\"alcohol_dataset.json\") #is not in collection format (hf to collection?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Dataset' object has no attribute 'generate'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m/Users/robertpraas/Desktop/ThoughtSource/notebooks/gpt_4_book.ipynb Cell 15\u001b[0m in \u001b[0;36m<cell line: 19>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/gpt_4_book.ipynb#X15sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39m# Configuration of the input and parameters of the language model \u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/gpt_4_book.ipynb#X15sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m config\u001b[39m=\u001b[39m{\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/gpt_4_book.ipynb#X15sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m     \u001b[39m# We compare three different prompts for the chain of thought generation:\u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/gpt_4_book.ipynb#X15sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m     \u001b[39m# \"Answer: Let's think step by step.\" and 'Answer: We should think about this step by step.', and \"Answer: First,\" \u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/gpt_4_book.ipynb#X15sZmlsZQ%3D%3D?line=15'>16</a>\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mwarn\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mTrue\u001b[39;00m,\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/gpt_4_book.ipynb#X15sZmlsZQ%3D%3D?line=16'>17</a>\u001b[0m }\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/gpt_4_book.ipynb#X15sZmlsZQ%3D%3D?line=18'>19</a>\u001b[0m alcohol_dataset\u001b[39m.\u001b[39;49mgenerate(config\u001b[39m=\u001b[39mconfig)\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Dataset' object has no attribute 'generate'"
     ]
    }
   ],
   "source": [
    "# Configuration of the input and parameters of the language model \n",
    "config={\n",
    "    # We compare three different prompts for the chain of thought generation:\n",
    "    # \"Answer: Let's think step by step.\" and 'Answer: We should think about this step by step.', and \"Answer: First,\" \n",
    "    \"cot_trigger_keys\": ['zhou_edited'],\n",
    "\n",
    "    # We use the same answer extraction prompt for all three prompts\n",
    "    # \"Therefore, among A through D, the answer is\"\n",
    "    \"answer_extraction_keys\": ['auto-kojima'], \n",
    "    \n",
    "    \"api_service\": \"openai\", # <--- Select which API you want to use\n",
    "    \"engine\": \"gpt-4\", # <--- Select which model you want to use\n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 512,\n",
    "    \"verbose\": False,\n",
    "    \"warn\": True,\n",
    "}\n",
    "\n",
    "alcohol.generate(config=config)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
